﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Text.RegularExpressions;
using System.IO;
using ICSharpCode.SharpZipLib.Zip;
using System.Collections.Specialized;
using Crawler.Model;

namespace Crawler.BusinessActions
{
    public class Utilities
    {
        private static string GetResponseString(WebResponse resp)
        {
            return GetResponseString(resp, Encoding.UTF8);
        }

        private static string GetResponseString(WebResponse resp, Encoding encoding)
        {
            StringReader txtRdr;

            using (Stream strm = resp.GetResponseStream())
            {
                StreamReader strmRdr = new StreamReader(strm, encoding);
                txtRdr = new StringReader(strmRdr.ReadToEnd());
            }

            //Dictionary<string, string> responseHeaders = new Dictionary<string, string>();

            //foreach (string key in resp.Headers.Keys)
            //{
            //    responseHeaders.Add(key, resp.Headers[key].ToString());
            //}
            //ResponseHeaders = responseHeaders;

            return txtRdr.ReadToEnd();
        }

        public static string GetPageHTML(string url)
        {
            string content = "";

            HttpWebRequest tmpWebRequest = WebRequest.Create(url) as HttpWebRequest;
            WebResponse tmpWebResponse = null;
            try
            {
                tmpWebResponse = tmpWebRequest.GetResponse();
                content = GetResponseString(tmpWebResponse);
            }
            catch
            {
            }
            finally
            {
                if (tmpWebResponse != null)
                {
                    tmpWebResponse.Close();
                }

            }
            return content;
        }

        public static string GetPageHTML(CaptureRuleInfo rule)
        {
            string content = GetPageHTML(rule.TaskItem.Url);
            content = GetFirstMatch(content, rule.StartString, rule.EndString);

            content = rule.Prefix + content + rule.Suffix;
            
            return content;
        }

        public static string GetPageHTML(string url, string startString, string endString, bool useRegex)
        {
            string content = "";
            string html = GetPageHTML(url);

            if (String.IsNullOrEmpty(startString) && String.IsNullOrEmpty(endString))
            {
                // 这里会自动执行finally中的关闭代码
                return html;
                //content = html;
            }

            if (useRegex)
            {
                #region 使用正则表达式
                string contentPattern = "";
                //Regex tmpRegex = null;
                //MatchCollection matches = null;
                if (String.IsNullOrEmpty(startString))
                {
                    contentPattern = @"(?<=)[\s\S]*?(?=" + endString + ")";
                }
                if (String.IsNullOrEmpty(endString))
                {
                    contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=)";
                }
                if (!String.IsNullOrEmpty(startString) && !String.IsNullOrEmpty(endString))
                {
                    contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=" + endString + ")";
                }
                content = GetFirstMatch(contentPattern, html);
                //tmpRegex = new Regex(contentPattern, RegexOptions.IgnoreCase);
                //matches = tmpRegex.Matches(html);
                //if (matches.Count > 0)
                //{
                //    content = matches[0].Value;
                //}
                #endregion
            }
            else
            {
                #region 不使用正则表达式,直接用字符串截取
                int startPosition = 0;
                int endPosition = html.Length - 1;
                if (!String.IsNullOrEmpty(startString))
                {
                    startPosition = html.IndexOf(startString);
                }
                if (!String.IsNullOrEmpty(endString))
                {
                    endPosition = html.LastIndexOf(endString) - 1;
                }

                content = html.Substring(startPosition, endPosition - startPosition);

                #endregion
            }
            
            return content;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <param name="startString"></param>
        /// <param name="endString"></param>
        /// <param name="useRegex"></param>
        /// <param name="title"></param>
        /// <returns></returns>
        public static string GetPageHTML(string url, string startString, string endString, bool useRegex, out string title)
        {
            string content = "";
            title = "";

            string html = GetPageHTML(url);

            title = GetFirstMatch(@"(?<=<title>)[\s\S]*?(?=</title>)", html);

            if (String.IsNullOrEmpty(startString) && String.IsNullOrEmpty(endString))
            {
                // 这里会自动执行finally中的关闭代码
                return html;
                //content = html;
            }

            if (useRegex)
            {
                #region 使用正则表达式
                string contentPattern = "";
                Regex tmpRegex = null;
                MatchCollection matches = null;
                if (String.IsNullOrEmpty(startString))
                {
                    contentPattern = @"(?<=)[\s\S]*?(?=" + endString + ")";
                }
                if (String.IsNullOrEmpty(endString))
                {
                    contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=)";
                }
                if (!String.IsNullOrEmpty(startString) && !String.IsNullOrEmpty(endString))
                {
                    contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=" + endString + ")";
                }
                tmpRegex = new Regex(contentPattern, RegexOptions.None);
                matches = tmpRegex.Matches(html);
                if (matches.Count > 0)
                {
                    content = matches[0].Value;
                }
                #endregion
            }
            else
            {
                #region 不使用正则表达式,直接用字符串截取
                int startPosition = 0;
                int endPosition = html.Length - 1;
                if (!String.IsNullOrEmpty(startString))
                {
                    startPosition = html.IndexOf(startString);
                }
                if (!String.IsNullOrEmpty(endString))
                {
                    endPosition = html.LastIndexOf(endString) - 1;
                }

                content = html.Substring(startPosition, endPosition - startPosition);

                #endregion
            }
            return content;
        }

        public static string GetFirstMatch(string s, string startString, string endString)
        {
            string content = "";
            string contentPattern = "";
            Regex tmpRegex = null;
            MatchCollection matches = null;
            if (String.IsNullOrEmpty(startString))
            {
                contentPattern = @"(?<=)[\s\S]*?(?=" + endString + ")";
            }
            if (String.IsNullOrEmpty(endString))
            {
                contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=)";
            }
            if (!String.IsNullOrEmpty(startString) && !String.IsNullOrEmpty(endString))
            {
                contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=" + endString + ")";
            }
            tmpRegex = new Regex(contentPattern, RegexOptions.IgnoreCase);
            matches = tmpRegex.Matches(s);
            if (matches.Count > 0)
            {
                content = matches[0].Value;
            }

            return content;
        }

        public static byte[] CompressHtml(string html, string url)
        {
            byte[] byData = null;
            if (html.Length == 0)
                return byData;
            using (MemoryStream ms = new MemoryStream())
            using (ZipOutputStream zip = new ZipOutputStream(ms))
            {
                zip.SetLevel(9); // 0 - store only to 9 - means best compression
                byte[] buffer = new byte[4096];

                #region 先压缩网页中的图片
                string imgPattern = @"(?<=<img )[\s\S]*?(?=>)";
                MatchCollection matches1 = GetMatches(imgPattern, html);
                StringCollection sc = new StringCollection();
                foreach (Match m in matches1)
                {
                    string imgSrcPattern = @"(?<=src[\s]*=[\s]*""?)[\s\S]*?(?=""? )";
                    //MatchCollection matches2 = GetMatches(imgPattern2, m.Value);
                    string imgSrc = GetFirstMatch(imgSrcPattern, m.Value);
                    if (imgSrc == null)
                        continue;

                    imgSrc = imgSrc.Trim().Trim('\"');
                    string imgName = imgSrc.Substring(imgSrc.LastIndexOf('/') + 1);
                    if (imgName.IndexOf("%20") > -1)
                    {
                        imgName = imgName.Replace("%20", "");
                    }
                    string fileName = imgName;
                    string tmpName = Guid.NewGuid().ToString("n");
                    if (imgName.IndexOfAny(System.IO.Path.GetInvalidFileNameChars()) != -1)
                    {
                         imgName = tmpName + ".jpg";
                        //continue;
                    }
                    if (string.IsNullOrEmpty(imgName))
                    {
                        imgName = tmpName;
                    }

                    if (!sc.Contains(imgName))
                    {
                        using (MemoryStream tmpStream = new MemoryStream())
                        {
                            try
                            {
                                string imgageExtension;
                                // 1
                                GetImageStream(tmpStream, imgSrc, out imgageExtension);
                                // 2
                                tmpStream.Position = 0;
                                // 3
                                if (fileName.IndexOfAny(System.IO.Path.GetInvalidFileNameChars()) != -1 || string.IsNullOrEmpty(fileName))
                                {
                                    imgName = tmpName + imgageExtension;
                                }
                                ZipEntry entry = new ZipEntry(imgName);
                                entry.DateTime = DateTime.Now;
                                // 4
                                zip.PutNextEntry(entry);
                                CopyStream(tmpStream, zip);
                            }
                            catch
                            {
                            }
                        }
                    }

                    imgSrc = FixUpUrl(url, imgSrc);
                    sc.Add(imgName);
                    html = html.Replace(imgSrc, imgName);
                }
                #endregion

                #region 再将网页内容压缩到MemoryStream流中
                ZipEntry zipEntry = new ZipEntry("ArticleContent.htm");
                zipEntry.DateTime = DateTime.Now;
                zip.PutNextEntry(zipEntry);
                using (MemoryStream memoryStream = StringToMemoryStream(html))
                {
                    int readCount;
                    while ((readCount = memoryStream.Read(buffer, 0, buffer.Length)) != 0)
                    {
                        zip.Write(buffer, 0, readCount);
                    }
                }
                #endregion

                zip.Flush();
                zip.Finish();

                byData = new Byte[ms.Length];
                //s.Position = 0;
                ms.Position = 0;
                ms.Read(byData, 0, byData.Length);

                zip.Close();
            }

            return byData;
        }

        public static string GetFirstMatch(string pattern, string originalText)
        {
            MatchCollection matches = GetMatches(pattern, originalText);
            if (matches.Count > 0)
            {
                return matches[0].Value;
            }
            else
            {
                return null;
            }
        }

        public static MatchCollection GetMatches(string pattern, string originalText)
        {
            Regex tmpRegex = new Regex(pattern, RegexOptions.IgnoreCase);
            MatchCollection matches = tmpRegex.Matches(originalText);

            return matches;
        }

        public static MatchCollection GetMatches(string startString, string endString, string originalText)
        {
            if (String.IsNullOrEmpty(startString) && String.IsNullOrEmpty(endString))
            {
                return null;
            }
            string contentPattern = "";
            if (String.IsNullOrEmpty(startString))
            {
                contentPattern = @"(?<=)[\s\S]*?(?=" + endString + ")";
            }
            if (String.IsNullOrEmpty(endString))
            {
                contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=)";
            }
            if (!String.IsNullOrEmpty(startString) && !String.IsNullOrEmpty(endString))
            {
                contentPattern = @"(?<=" + startString + @")[\s\S]*?(?=" + endString + ")";
            }

            Regex tmpRegex = new Regex(contentPattern, RegexOptions.IgnoreCase);
            MatchCollection matches = tmpRegex.Matches(originalText);

            return matches;
        }

        /// <summary>
        /// 将从网络上下载的图片响应流，复制到另一个临时流中
        /// </summary>
        /// <param name="to"></param>
        /// <param name="imgSrc"></param>
        private static void GetImageStream(Stream to, string imgSrc, out string imgageExtension)
        {
            imgageExtension = "";

            WebRequest request = WebRequest.Create(imgSrc);
            WebResponse response = null;

            try
            {
                response = request.GetResponse();
                using (Stream from = response.GetResponseStream())
                {
                    imgageExtension = response.Headers["Content-Type"];
                    imgageExtension = GetImageExtension(imgageExtension);

                    int readCount;
                    byte[] buffer = new byte[4096];
                    while ((readCount = from.Read(buffer, 0, buffer.Length)) != 0)
                    {
                        to.Write(buffer, 0, readCount);
                    }
                }
            }
            catch (Exception ex)
            {
                string s = ex.Message;
            }
            finally
            {
                response.Close();
            }
            //if (to.CanSeek)
            //{
            //    to.Position = 0;
            //}
        }

        /// <summary>
        /// 将字符串转换为 MemoryStream.
        /// </summary>
        static MemoryStream StringToMemoryStream(string s)
        {
            byte[] buffer = System.Text.Encoding.UTF8.GetBytes(s);
            return new System.IO.MemoryStream(buffer);
        }

        /// <summary>
        /// 将 MemoryStream 转换为. Makes some assumptions about the content of the stream. 
        /// </summary>
        /// <param name="s"></param>
        /// <returns></returns>
        static String MemoryStreamToString(MemoryStream ms)
        {
            byte[] byteArray = ms.ToArray();
            return System.Text.Encoding.UTF8.GetString(byteArray);
        }

        /// <summary>
        /// 流复制
        /// </summary>
        /// <param name="src"></param>
        /// <param name="dest"></param>
        static void CopyStream(System.IO.Stream from, System.IO.Stream to)
        {
            byte[] buffer = new byte[4096];
            int len = from.Read(buffer, 0, buffer.Length);
            while (len > 0)
            {
                to.Write(buffer, 0, len);
                len = from.Read(buffer, 0, buffer.Length);
            }
            to.Flush();

            //int readCount;
            //byte[] buffer = new byte[1024];

            //while ((readCount = from.Read(buffer, 0, buffer.Length)) != 0)
            //{
            //    to.Write(buffer, 0, readCount);
            //}

            //if (from.CanSeek)
            //{
            //    from.Position = 0;
            //}

        }

        public static string FixUpUrl(string baseUrl, string originalUrl)
        {

            if (originalUrl == null || originalUrl.Length == 0)
                return originalUrl;
            if (baseUrl == null || baseUrl.Length == 0)
                return originalUrl;
            if (originalUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
                return originalUrl;

            string tmp = baseUrl.Substring(0, baseUrl.LastIndexOf('/'));

            if (originalUrl[0] == '/')
            {
                if (baseUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase))
                {
                    return baseUrl.Substring(0, baseUrl.IndexOf('/', 7) ) + originalUrl;
                }
                else
                {
                    return "http://" + baseUrl.Substring(0, baseUrl.IndexOf('/')) + originalUrl;
                }
            }

            if (originalUrl[0] == '.')
            {
                if (originalUrl[1] == '.')
                {
                    while (originalUrl.StartsWith("../"))
                    {
                        originalUrl = originalUrl.Substring(3);
                        tmp = tmp.Substring(0, tmp.LastIndexOf('/') );
                    }
                    return tmp + "/" + originalUrl;
                }
                else
                {
                    return tmp + originalUrl.Substring(1);
                }
            }


            return tmp + "/" + originalUrl;
        }

        private string GetPageContent(string url)
        {
            string content = "";

            HttpWebRequest tmpWebRequest = WebRequest.Create(url) as HttpWebRequest;
            WebResponse tmpWebResponse = null;
            try
            {
                tmpWebResponse = tmpWebRequest.GetResponse();
                string html = GetResponseString(tmpWebResponse);

                string contentPattern = @"(?<=<div class[\s]*=[\s]*""post""[\s]*>)[\s\S]*?(?=<div id=""post_next_prev"">)";
                Regex tmpRegex = new Regex(contentPattern, RegexOptions.IgnoreCase);
                MatchCollection matches = tmpRegex.Matches(html);
                if (matches.Count > 0)
                {
                    content = matches[0].Value;
                }
            }
            catch
            {

            }
            finally
            {
                if (tmpWebResponse != null)
                {
                    tmpWebResponse.Close();
                }
            }
            return content;
        }

        private static string GetImageExtension(string s)
        {
            switch (s.ToLower())
            {
                case "image/bmp":
                    return ".bmp";
                case "image/cis-cod":
                    return ".cod";
                case "image/gif":
                    return ".gif";
                case "image/x-png":
                case "image/png":
                    return ".png";
                case "image/ief":
                    return ".ief";
                case "image/pjpeg":
                case "image/jpeg":
                    return ".jpg";
                case "image/pipeg":
                    return ".jfif";
                case "image/tiff":
                    return ".tif";
                case "image/svg+xml":
                    return ".svg";
                case "image/x-icon":
                    return ".ico";

                default:
                    return ".jpg";
            }

			//通过判断文件内容的头2个字节可以确定文件格式  255216是jpg //7173是gif //6677是BMP //13780是PNG

            //if (imgName.IndexOf('?') > 0)
            //{
            //    imgName = imgName.Substring(0, imgName.IndexOf('?'));
            //    if (imgName != null && imgName.Length > 0)
            //    {
            //        if (imgName.IndexOf('.') > 0)
            //        {
            //            imgName = imgName.Substring(0, imgName.IndexOf('.'));
            //        }
            //        else
            //        {
            //            imgName = Guid.NewGuid().ToString("n");
            //        }
            //    }
            //    else
            //    {
            //        imgName = Guid.NewGuid().ToString("n");
            //    }
            //    imgName = imgName + ".jpg";
            //}
            //else
            //{
            //    imgName = Guid.NewGuid().ToString("n") + ".jpg";
            //}
        }
    }


}
